{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8fbee965",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-04-16T13:52:21.043963Z",
     "start_time": "2024-04-16T13:52:21.036711Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# setting the environment variables\n",
    "import sys\n",
    "import os\n",
    "\n",
    "sys.path.insert(0, os.path.abspath('..'))\n",
    "\n",
    "from config import set_environment\n",
    "set_environment()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf636e2d",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## Evaluating pass@k performance of code solutions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ad68f80d",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# pip install datasets evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a84ef871-2ff6-43fc-a033-a5c5e1903cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from evaluate import load\n",
    "\n",
    "human_eval = load_dataset(\"openai_humaneval\")\n",
    "code_eval_metric = load(\"code_eval\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4dc3f31-4172-4bf0-83bd-d470c0c9c956",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_cases = [\"assert add(2,3)==5\"]\n",
    "candidates = [[\"def add(a,b): return a*b\", \"def add(a, b): return a+b\"]]\n",
    "pass_at_k, results = code_eval_metric.compute(references=test_cases, predictions=candidates, k=[1, 2])\n",
    "print(pass_at_k)  # {'pass@1': 0.5, 'pass@2': 1.0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9262a67",
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-04-16T14:07:41.254202Z"
    },
    "collapsed": false,
    "is_executing": true,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from langchain_google_vertexai import VertexAI\n",
    "\n",
    "question = \"\"\"\n",
    "Given an integer n, return a string array answer (1-indexed) where:\n",
    "\n",
    "answer[i] == \"FizzBuzz\" if i is divisible by 3 and 5.\n",
    "answer[i] == \"Fizz\" if i is divisible by 3.\n",
    "answer[i] == \"Buzz\" if i is divisible by 5.\n",
    "answer[i] == i (as a string) if none of the above conditions are true.\n",
    "\"\"\"\n",
    "\n",
    "llm = VertexAI(model_name=\"code-bison\")\n",
    "print(llm(question))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e6d4133",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
